{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0b1033d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import ipynbname\n",
    "from torchtext.datasets import AG_NEWS\n",
    "from torch.utils.data.dataset import random_split\n",
    "# notebook_path = ipynbname.path()\n",
    "# notebook_path\n",
    "DATA_DIR = r'C:\\Users\\caofei\\Desktop\\torch1\\案例\\新闻分类\\datasets\\AG News\\data'\n",
    "train_datasets, test_datasets = AG_NEWS(root=DATA_DIR, split=('train', 'test'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "957d95cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, \"Wall St. Bears Claw Back Into the Black (Reuters) Reuters - Short-sellers, Wall Street's dwindling\\\\band of ultra-cynics, are seeing green again.\")\n",
      "单行文本长度 2\n",
      "第一个值 3\n",
      "第二个值 Carlyle Looks Toward Commercial Aerospace (Reuters) Reuters - Private investment firm Carlyle Group,\\which has a reputation for making well-timed and occasionally\\controversial plays in the defense industry, has quietly placed\\its bets on another part of the market.\n",
      "第一个值 3\n",
      "第二个值 Oil and Economy Cloud Stocks' Outlook (Reuters) Reuters - Soaring crude prices plus worries\\about the economy and the outlook for earnings are expected to\\hang over the stock market next week during the depth of the\\summer doldrums.\n",
      "第一个值 3\n",
      "第二个值 Iraq Halts Oil Exports from Main Southern Pipeline (Reuters) Reuters - Authorities have halted oil export\\flows from the main pipeline in southern Iraq after\\intelligence showed a rebel militia could strike\\infrastructure, an oil official said on Saturday.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\VirtualProject\\Python37Env\\torch_py38\\lib\\site-packages\\torch\\utils\\data\\datapipes\\iter\\combining.py:333: UserWarning: Some child DataPipes are not exhausted when __iter__ is called. We are resetting the buffer and each child DataPipe will read from the start again.\n",
      "  warnings.warn(\"Some child DataPipes are not exhausted when __iter__ is called. We are resetting \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[\"Wall St. Bears Claw Back Into the Black (Reuters) Reuters - Short-sellers, Wall Street's dwindling\\\\band of ultra-cynics, are seeing green again.\",\n",
       " 'Carlyle Looks Toward Commercial Aerospace (Reuters) Reuters - Private investment firm Carlyle Group,\\\\which has a reputation for making well-timed and occasionally\\\\controversial plays in the defense industry, has quietly placed\\\\its bets on another part of the market.',\n",
       " \"Oil and Economy Cloud Stocks' Outlook (Reuters) Reuters - Soaring crude prices plus worries\\\\about the economy and the outlook for earnings are expected to\\\\hang over the stock market next week during the depth of the\\\\summer doldrums.\",\n",
       " 'Iraq Halts Oil Exports from Main Southern Pipeline (Reuters) Reuters - Authorities have halted oil export\\\\flows from the main pipeline in southern Iraq after\\\\intelligence showed a rebel militia could strike\\\\infrastructure, an oil official said on Saturday.']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S=[]\n",
    "for index,_ in enumerate(train_datasets):\n",
    "    S.append(_[1])\n",
    "    if index == 0:\n",
    "        print(_)\n",
    "        print('单行文本长度',len(_))\n",
    "    else:\n",
    "        print('第一个值',_[0])\n",
    "        print('第二个值',_[1])\n",
    "    if index==3:\n",
    "        break\n",
    "    # break\n",
    "S"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c2b80ca9",
   "metadata": {},
   "outputs": [],
   "source": [
    "cutlen = 64\n",
    "from keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86b26247",
   "metadata": {},
   "source": [
    "Tokenizer api参考地址\n",
    "\n",
    "https://tensorflow.google.cn/api_docs/python/tf/keras/preprocessing/text/Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4faf21e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Wall St. Bears Claw Back Into the Black (Reuters) Reuters - Short-sellers, Wall Street's dwindling\\\\band of ultra-cynics, are seeing green again.\",\n",
       " 'Carlyle Looks Toward Commercial Aerospace (Reuters) Reuters - Private investment firm Carlyle Group,\\\\which has a reputation for making well-timed and occasionally\\\\controversial plays in the defense industry, has quietly placed\\\\its bets on another part of the market.',\n",
       " \"Oil and Economy Cloud Stocks' Outlook (Reuters) Reuters - Soaring crude prices plus worries\\\\about the economy and the outlook for earnings are expected to\\\\hang over the stock market next week during the depth of the\\\\summer doldrums.\",\n",
       " 'Iraq Halts Oil Exports from Main Southern Pipeline (Reuters) Reuters - Authorities have halted oil export\\\\flows from the main pipeline in southern Iraq after\\\\intelligence showed a rebel militia could strike\\\\infrastructure, an oil official said on Saturday.']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "76528ad2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'the': 1, 'reuters': 2, 'oil': 3, 'of': 4, 'and': 5, 'wall': 6, 'are': 7, 'carlyle': 8, 'has': 9, 'a': 10, 'for': 11, 'in': 12, 'on': 13, 'market': 14, 'economy': 15, 'outlook': 16, 'iraq': 17, 'from': 18, 'main': 19, 'southern': 20, 'pipeline': 21, 'st': 22, 'bears': 23, 'claw': 24, 'back': 25, 'into': 26, 'black': 27, 'short': 28, 'sellers': 29, \"street's\": 30, 'dwindling': 31, 'band': 32, 'ultra': 33, 'cynics': 34, 'seeing': 35, 'green': 36, 'again': 37, 'looks': 38, 'toward': 39, 'commercial': 40, 'aerospace': 41, 'private': 42, 'investment': 43, 'firm': 44, 'group': 45, 'which': 46, 'reputation': 47, 'making': 48, 'well': 49, 'timed': 50, 'occasionally': 51, 'controversial': 52, 'plays': 53, 'defense': 54, 'industry': 55, 'quietly': 56, 'placed': 57, 'its': 58, 'bets': 59, 'another': 60, 'part': 61, 'cloud': 62, \"stocks'\": 63, 'soaring': 64, 'crude': 65, 'prices': 66, 'plus': 67, 'worries': 68, 'about': 69, 'earnings': 70, 'expected': 71, 'to': 72, 'hang': 73, 'over': 74, 'stock': 75, 'next': 76, 'week': 77, 'during': 78, 'depth': 79, 'summer': 80, 'doldrums': 81, 'halts': 82, 'exports': 83, 'authorities': 84, 'have': 85, 'halted': 86, 'export': 87, 'flows': 88, 'after': 89, 'intelligence': 90, 'showed': 91, 'rebel': 92, 'militia': 93, 'could': 94, 'strike': 95, 'infrastructure': 96, 'an': 97, 'official': 98, 'said': 99, 'saturday': 100}\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer()\n",
    "all_datasets_texts = S\n",
    "tokenizer.fit_on_texts(all_datasets_texts)\n",
    "# display(tokenizer.word_index)\n",
    "print(tokenizer.word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b589bc90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "单词索引： {'the': 1, 'reuters': 2, 'oil': 3, 'of': 4, 'and': 5, 'wall': 6, 'are': 7, 'carlyle': 8, 'has': 9, 'a': 10, 'for': 11, 'in': 12, 'on': 13, 'market': 14, 'economy': 15, 'outlook': 16, 'iraq': 17, 'from': 18, 'main': 19, 'southern': 20, 'pipeline': 21, 'st': 22, 'bears': 23, 'claw': 24, 'back': 25, 'into': 26, 'black': 27, 'short': 28, 'sellers': 29, \"street's\": 30, 'dwindling': 31, 'band': 32, 'ultra': 33, 'cynics': 34, 'seeing': 35, 'green': 36, 'again': 37, 'looks': 38, 'toward': 39, 'commercial': 40, 'aerospace': 41, 'private': 42, 'investment': 43, 'firm': 44, 'group': 45, 'which': 46, 'reputation': 47, 'making': 48, 'well': 49, 'timed': 50, 'occasionally': 51, 'controversial': 52, 'plays': 53, 'defense': 54, 'industry': 55, 'quietly': 56, 'placed': 57, 'its': 58, 'bets': 59, 'another': 60, 'part': 61, 'cloud': 62, \"stocks'\": 63, 'soaring': 64, 'crude': 65, 'prices': 66, 'plus': 67, 'worries': 68, 'about': 69, 'earnings': 70, 'expected': 71, 'to': 72, 'hang': 73, 'over': 74, 'stock': 75, 'next': 76, 'week': 77, 'during': 78, 'depth': 79, 'summer': 80, 'doldrums': 81, 'halts': 82, 'exports': 83, 'authorities': 84, 'have': 85, 'halted': 86, 'export': 87, 'flows': 88, 'after': 89, 'intelligence': 90, 'showed': 91, 'rebel': 92, 'militia': 93, 'could': 94, 'strike': 95, 'infrastructure': 96, 'an': 97, 'official': 98, 'said': 99, 'saturday': 100}\n"
     ]
    }
   ],
   "source": [
    "# 获取单词索引\n",
    "word_index = tokenizer.word_index\n",
    "print(\"单词索引：\", word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7f2a2af6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Wall St. Bears Claw Back Into the Black (Reuters) Reuters - Short-sellers, Wall Street's dwindling\\\\band of ultra-cynics, are seeing green again.\""
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1a831efd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_index['wall']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b742db31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6, 22, 23, 24, 25, 26, 1, 27, 2, 2, 28, 29, 6, 30, 31, 32, 4, 33, 34, 7, 35, 36, 37], [8, 38, 39, 40, 41, 2, 2, 42, 43, 44, 8, 45, 46, 9, 10, 47, 11, 48, 49, 50, 5, 51, 52, 53, 12, 1, 54, 55, 9, 56, 57, 58, 59, 13, 60, 61, 4, 1, 14], [3, 5, 15, 62, 63, 16, 2, 2, 64, 65, 66, 67, 68, 69, 1, 15, 5, 1, 16, 11, 70, 7, 71, 72, 73, 74, 1, 75, 14, 76, 77, 78, 1, 79, 4, 1, 80, 81], [17, 82, 3, 83, 18, 19, 20, 21, 2, 2, 84, 85, 86, 3, 87, 88, 18, 1, 19, 21, 12, 20, 17, 89, 90, 91, 10, 92, 93, 94, 95, 96, 97, 3, 98, 99, 13, 100]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[[6,\n",
       "  22,\n",
       "  23,\n",
       "  24,\n",
       "  25,\n",
       "  26,\n",
       "  1,\n",
       "  27,\n",
       "  2,\n",
       "  2,\n",
       "  28,\n",
       "  29,\n",
       "  6,\n",
       "  30,\n",
       "  31,\n",
       "  32,\n",
       "  4,\n",
       "  33,\n",
       "  34,\n",
       "  7,\n",
       "  35,\n",
       "  36,\n",
       "  37]]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenizer.texts_to_sequences(S[0])\n",
    "# 将文本转换为序列\n",
    "sequences = tokenizer.texts_to_sequences(S)\n",
    "print(sequences)\n",
    "tokenizer.texts_to_sequences([S[0]])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53f18168",
   "metadata": {},
   "source": [
    "# tf.keras.utils.pad_sequences\n",
    "https://tensorflow.google.cn/api_docs/python/tf/keras/utils/pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a96492f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1],\n",
       "       [2, 3],\n",
       "       [5, 6]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 2, 3],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 4, 5, 6]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sequence1 = [[1], [2, 3], [4, 5, 6]]\n",
    "from keras.preprocessing import sequence\n",
    "display(sequence.pad_sequences(sequence1,maxlen=2))\n",
    "display(sequence.pad_sequences(sequence1,maxlen=10))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e2d937a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfc80867",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "单词索引： {'sample': 1, 'sentence': 2, 'this': 3, 'is': 4, 'a': 5, 'another': 6, 'here': 7}\n",
      "整数序列： [[3, 4, 5, 1, 2], [6, 1, 2, 7]]\n",
      "转换回的文本： ['this is a sample sentence', 'another sample sentence here']\n",
      "文本矩阵： [[0. 1. 1. ... 0. 0. 0.]\n",
      " [0. 1. 1. ... 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "# 案例1\n",
    "\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "\n",
    "# 初始化 Tokenizer 对象，保留最常见的 1000 个单词\n",
    "tokenizer = Tokenizer(num_words=1000)\n",
    "\n",
    "# 示例文本数据\n",
    "texts = [\"This is a sample sentence.\", \"Another sample sentence here.\"]\n",
    "\n",
    "# 拟合文本数据\n",
    "tokenizer.fit_on_texts(texts)\n",
    "\n",
    "# 获取单词索引\n",
    "word_index = tokenizer.word_index\n",
    "print(\"单词索引：\", word_index)\n",
    "\n",
    "# 将文本转换为序列\n",
    "sequences = tokenizer.texts_to_sequences(texts)\n",
    "print(\"整数序列：\", sequences)\n",
    "\n",
    "# 将序列转换为文本\n",
    "texts_from_sequences = tokenizer.sequences_to_texts(sequences)\n",
    "print(\"转换回的文本：\", texts_from_sequences)\n",
    "\n",
    "# 将文本转换为矩阵，使用二进制模式\n",
    "matrix = tokenizer.texts_to_matrix(texts, mode='binary')\n",
    "print(\"文本矩阵：\", matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d562e413",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "单词索引： {'i': 1, 'hava': 2, 'a': 3, 'apple': 4, 'sdfad': 5, 'pencil': 6, 'box': 7}\n",
      "整数序列： [[1], [2], [3], [4], [5], [6, 7]]\n",
      "转换回的文本： ['i', 'hava', 'a', 'apple', 'sdfad', 'pencil box']\n",
      "文本矩阵： [[0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 1. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "# 案例2\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "\n",
    "# 初始化 Tokenizer 对象，保留最常见的 1000 个单词\n",
    "tokenizer = Tokenizer(num_words=1000)\n",
    "\n",
    "# 示例文本数据\n",
    "texts = ['i','hava','a','apple','sdfad','pencil box']\n",
    "\n",
    "# 拟合文本数据\n",
    "tokenizer.fit_on_texts(texts)\n",
    "\n",
    "# 获取单词索引\n",
    "word_index = tokenizer.word_index\n",
    "print(\"单词索引：\", word_index)\n",
    "\n",
    "# 将文本转换为序列\n",
    "sequences = tokenizer.texts_to_sequences(texts)\n",
    "print(\"整数序列：\", sequences)\n",
    "\n",
    "# 将序列转换为文本\n",
    "texts_from_sequences = tokenizer.sequences_to_texts(sequences)\n",
    "print(\"转换回的文本：\", texts_from_sequences)\n",
    "\n",
    "# 将文本转换为矩阵，使用二进制模式\n",
    "matrix = tokenizer.texts_to_matrix(texts, mode='binary')\n",
    "print(\"文本矩阵：\", matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "5a61ee30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.8189, -0.2009, -1.0997, -0.2507,  0.8162],\n",
       "        [ 1.8122,  0.1314, -0.6540, -1.1265,  0.2132],\n",
       "        [-0.7360,  0.5985, -2.5146, -1.1089, -0.6534]],\n",
       "       grad_fn=<EmbeddingBackward0>)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "embedding = nn.Embedding(300,5,sparse=True)\n",
    "embedding(torch.tensor([1,2,3]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "cf5f2f89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.8189, -0.2009, -1.0997, -0.2507,  0.8162],\n",
       "         [ 1.8122,  0.1314, -0.6540, -1.1265,  0.2132],\n",
       "         [-0.7360,  0.5985, -2.5146, -1.1089, -0.6534]],\n",
       "\n",
       "        [[-0.8189, -0.2009, -1.0997, -0.2507,  0.8162],\n",
       "         [ 1.8122,  0.1314, -0.6540, -1.1265,  0.2132],\n",
       "         [-0.7360,  0.5985, -2.5146, -1.1089, -0.6534]],\n",
       "\n",
       "        [[-0.8189, -0.2009, -1.0997, -0.2507,  0.8162],\n",
       "         [ 1.8122,  0.1314, -0.6540, -1.1265,  0.2132],\n",
       "         [-0.7360,  0.5985, -2.5146, -1.1089, -0.6534]]],\n",
       "       grad_fn=<EmbeddingBackward0>)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 3, 5])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(embedding(torch.tensor([[1,2,3],[1,2,3],[1,2,3]])))\n",
    "display(embedding(torch.tensor([[1,2,3],[1,2,3],[1,2,3]])).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbffa75e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch_py38",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
